{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import gc\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cur time = 2018/09/21 18:54:08\n",
      "(13887, 3251) (12955, 3251)\n",
      "cur time = 2018/09/21 18:54:08\n"
     ]
    }
   ],
   "source": [
    "print('cur time = ' + str(datetime.datetime.now().strftime(\"%Y/%m/%d %H:%M:%S\")))\n",
    "train = np.load('../X_train.npy')\n",
    "test = np.load('../X_test.npy')\n",
    "train_labels = np.load('../labels.npy')\n",
    "print train.shape,test.shape\n",
    "print('cur time = ' + str(datetime.datetime.now().strftime(\"%Y/%m/%d %H:%M:%S\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(train_labels, n_folds=5, shuffle=True, random_state=42)\n",
    "\n",
    "meta_train = np.zeros(shape = (len(train),8))\n",
    "meta_test = np.zeros(shape = (len(test),8))\n",
    "\n",
    "for i,(tr_ind,te_ind) in enumerate(skf):\n",
    "    print 'FOLD: ',i\n",
    "    print len(te_ind),len(tr_ind)\n",
    "    X_train,X_train_label = train[tr_ind],train_labels[tr_ind]\n",
    "    X_val,X_val_label = train[te_ind],train_labels[te_ind]\n",
    "    dtrain = lgb.Dataset(X_train,X_train_label) \n",
    "    dval   = lgb.Dataset(X_val,X_val_label, reference = dtrain)   \n",
    "    params = {\n",
    "            'task':'train', \n",
    "            'boosting_type':'gbdt',\n",
    "            'num_leaves': 15,\n",
    "            'objective': 'multiclass',\n",
    "            'num_class':8,\n",
    "            'learning_rate': 0.05,\n",
    "            'feature_fraction': 0.85,\n",
    "            'subsample':0.85,\n",
    "            'num_threads': 32,\n",
    "            'metric':'multi_logloss',\n",
    "            'seed':100\n",
    "        }  \n",
    "    model = lgb.train(params, dtrain, num_boost_round=100000,valid_sets=[dtrain,dval],verbose_eval=100, early_stopping_rounds=100)  \n",
    "    pred_val = model.predict(X_val)\n",
    "    pred_test = model.predict(test)\n",
    "    \n",
    "    meta_train[te_ind] = pred_val\n",
    "    meta_test += pred_test\n",
    "meta_test /= 5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(meta_train,'../train_meta_lgb_1.pkl')\n",
    "pd.to_pickle(meta_test,'../test_meta_lgb_1.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOLD:  0\n",
      "2780 11107\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.105693\tvalid_1's multi_logloss: 0.290438\n",
      "[200]\ttraining's multi_logloss: 0.0243107\tvalid_1's multi_logloss: 0.28446\n",
      "Early stopping, best iteration is:\n",
      "[145]\ttraining's multi_logloss: 0.0517928\tvalid_1's multi_logloss: 0.277273\n",
      "FOLD:  1\n",
      "2779 11108\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.108126\tvalid_1's multi_logloss: 0.284527\n",
      "[200]\ttraining's multi_logloss: 0.0254294\tvalid_1's multi_logloss: 0.283195\n",
      "Early stopping, best iteration is:\n",
      "[139]\ttraining's multi_logloss: 0.0583621\tvalid_1's multi_logloss: 0.273231\n",
      "FOLD:  2\n",
      "2777 11110\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.107591\tvalid_1's multi_logloss: 0.271276\n",
      "[200]\ttraining's multi_logloss: 0.0256978\tvalid_1's multi_logloss: 0.267876\n",
      "Early stopping, best iteration is:\n",
      "[151]\ttraining's multi_logloss: 0.0490566\tvalid_1's multi_logloss: 0.258754\n",
      "FOLD:  3\n",
      "2776 11111\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.109872\tvalid_1's multi_logloss: 0.2752\n",
      "[200]\ttraining's multi_logloss: 0.0267958\tvalid_1's multi_logloss: 0.266528\n",
      "Early stopping, best iteration is:\n",
      "[153]\ttraining's multi_logloss: 0.0492415\tvalid_1's multi_logloss: 0.260417\n",
      "FOLD:  4\n",
      "2775 11112\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.108239\tvalid_1's multi_logloss: 0.286993\n",
      "[200]\ttraining's multi_logloss: 0.0260953\tvalid_1's multi_logloss: 0.276078\n",
      "Early stopping, best iteration is:\n",
      "[155]\ttraining's multi_logloss: 0.0471788\tvalid_1's multi_logloss: 0.270497\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold(train_labels, n_folds=5, shuffle=True, random_state=42)\n",
    "\n",
    "meta_train = np.zeros(shape = (len(train),8))\n",
    "meta_test = np.zeros(shape = (len(test),8))\n",
    "\n",
    "for i,(tr_ind,te_ind) in enumerate(skf):\n",
    "    print 'FOLD: ',i\n",
    "    print len(te_ind),len(tr_ind)\n",
    "    X_train,X_train_label = train[tr_ind],train_labels[tr_ind]\n",
    "    X_val,X_val_label = train[te_ind],train_labels[te_ind]\n",
    "    dtrain = lgb.Dataset(X_train,X_train_label) \n",
    "    dval   = lgb.Dataset(X_val,X_val_label, reference = dtrain)   \n",
    "    params = {\n",
    "            'task':'train', \n",
    "            'boosting_type':'gbdt',\n",
    "            'num_leaves': 31,\n",
    "            'objective': 'multiclass',\n",
    "            'num_class':8,\n",
    "            'learning_rate': 0.05,\n",
    "            'feature_fraction': 0.85,\n",
    "            'subsample':0.85,\n",
    "            'num_threads': 32,\n",
    "            'metric':'multi_logloss',\n",
    "            'seed':100\n",
    "        }  \n",
    "    model = lgb.train(params, dtrain, num_boost_round=100000,valid_sets=[dtrain,dval],verbose_eval=100, early_stopping_rounds=100)  \n",
    "    pred_val = model.predict(X_val)\n",
    "    pred_test = model.predict(test)\n",
    "    \n",
    "    meta_train[te_ind] = pred_val\n",
    "    meta_test += pred_test\n",
    "meta_test /= 5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(meta_train,'../train_meta_lgb_2.pkl')\n",
    "pd.to_pickle(meta_test,'../test_meta_lgb_2.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOLD:  0\n",
      "2780 11107\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.126813\tvalid_1's multi_logloss: 0.299223\n",
      "[200]\ttraining's multi_logloss: 0.0319222\tvalid_1's multi_logloss: 0.278803\n",
      "Early stopping, best iteration is:\n",
      "[161]\ttraining's multi_logloss: 0.0520005\tvalid_1's multi_logloss: 0.276196\n",
      "FOLD:  1\n",
      "2779 11108\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.128834\tvalid_1's multi_logloss: 0.292494\n",
      "[200]\ttraining's multi_logloss: 0.0332951\tvalid_1's multi_logloss: 0.277843\n",
      "Early stopping, best iteration is:\n",
      "[153]\ttraining's multi_logloss: 0.0597567\tvalid_1's multi_logloss: 0.272742\n",
      "FOLD:  2\n",
      "2777 11110\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.128497\tvalid_1's multi_logloss: 0.279648\n",
      "[200]\ttraining's multi_logloss: 0.0334364\tvalid_1's multi_logloss: 0.263845\n",
      "Early stopping, best iteration is:\n",
      "[159]\ttraining's multi_logloss: 0.0551787\tvalid_1's multi_logloss: 0.25859\n",
      "FOLD:  3\n",
      "2776 11111\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.130386\tvalid_1's multi_logloss: 0.286192\n",
      "[200]\ttraining's multi_logloss: 0.0347223\tvalid_1's multi_logloss: 0.263253\n",
      "Early stopping, best iteration is:\n",
      "[169]\ttraining's multi_logloss: 0.0501232\tvalid_1's multi_logloss: 0.260649\n",
      "FOLD:  4\n",
      "2775 11112\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.129009\tvalid_1's multi_logloss: 0.296055\n",
      "[200]\ttraining's multi_logloss: 0.0340881\tvalid_1's multi_logloss: 0.274158\n",
      "Early stopping, best iteration is:\n",
      "[173]\ttraining's multi_logloss: 0.0469372\tvalid_1's multi_logloss: 0.272973\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold(train_labels, n_folds=5, shuffle=True, random_state=42)\n",
    "\n",
    "meta_train = np.zeros(shape = (len(train),8))\n",
    "meta_test = np.zeros(shape = (len(test),8))\n",
    "\n",
    "for i,(tr_ind,te_ind) in enumerate(skf):\n",
    "    print 'FOLD: ',i\n",
    "    print len(te_ind),len(tr_ind)\n",
    "    X_train,X_train_label = train[tr_ind],train_labels[tr_ind]\n",
    "    X_val,X_val_label = train[te_ind],train_labels[te_ind]\n",
    "    dtrain = lgb.Dataset(X_train,X_train_label) \n",
    "    dval   = lgb.Dataset(X_val,X_val_label, reference = dtrain)   \n",
    "    params = {\n",
    "            'task':'train', \n",
    "            'boosting_type':'gbdt',\n",
    "            'num_leaves': 31,\n",
    "            'objective': 'multiclass',\n",
    "            'num_class':8,\n",
    "            'learning_rate': 0.045,\n",
    "            'feature_fraction': 0.8,\n",
    "            'subsample':0.8,\n",
    "            'num_threads': 32,\n",
    "            'metric':'multi_logloss',\n",
    "            'seed':100\n",
    "        }  \n",
    "    model = lgb.train(params, dtrain, num_boost_round=100000,valid_sets=[dtrain,dval],verbose_eval=100, early_stopping_rounds=100)  \n",
    "    pred_val = model.predict(X_val)\n",
    "    pred_test = model.predict(test)\n",
    "    \n",
    "    meta_train[te_ind] = pred_val\n",
    "    meta_test += pred_test\n",
    "meta_test /= 5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(meta_train,'../train_meta_lgb_3.pkl')\n",
    "pd.to_pickle(meta_test,'../test_meta_lgb_3.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FOLD:  0\n",
      "2780 11107\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.0771172\tvalid_1's multi_logloss: 0.289138\n",
      "[200]\ttraining's multi_logloss: 0.00851115\tvalid_1's multi_logloss: 0.298243\n",
      "Early stopping, best iteration is:\n",
      "[133]\ttraining's multi_logloss: 0.0357694\tvalid_1's multi_logloss: 0.27818\n",
      "FOLD:  1\n",
      "2779 11108\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.0780999\tvalid_1's multi_logloss: 0.289059\n",
      "[200]\ttraining's multi_logloss: 0.00887645\tvalid_1's multi_logloss: 0.298286\n",
      "Early stopping, best iteration is:\n",
      "[134]\ttraining's multi_logloss: 0.0357742\tvalid_1's multi_logloss: 0.278663\n",
      "FOLD:  2\n",
      "2777 11110\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.0784245\tvalid_1's multi_logloss: 0.274011\n",
      "[200]\ttraining's multi_logloss: 0.00891692\tvalid_1's multi_logloss: 0.282485\n",
      "Early stopping, best iteration is:\n",
      "[134]\ttraining's multi_logloss: 0.0356565\tvalid_1's multi_logloss: 0.263027\n",
      "FOLD:  3\n",
      "2776 11111\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.0795669\tvalid_1's multi_logloss: 0.280272\n",
      "[200]\ttraining's multi_logloss: 0.00927117\tvalid_1's multi_logloss: 0.284248\n",
      "Early stopping, best iteration is:\n",
      "[135]\ttraining's multi_logloss: 0.0357068\tvalid_1's multi_logloss: 0.267277\n",
      "FOLD:  4\n",
      "2775 11112\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's multi_logloss: 0.0782005\tvalid_1's multi_logloss: 0.287082\n",
      "[200]\ttraining's multi_logloss: 0.00896856\tvalid_1's multi_logloss: 0.294814\n",
      "Early stopping, best iteration is:\n",
      "[129]\ttraining's multi_logloss: 0.0400827\tvalid_1's multi_logloss: 0.277252\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold(train_labels, n_folds=5, shuffle=True, random_state=42)\n",
    "\n",
    "meta_train = np.zeros(shape = (len(train),8))\n",
    "meta_test = np.zeros(shape = (len(test),8))\n",
    "\n",
    "for i,(tr_ind,te_ind) in enumerate(skf):\n",
    "    print 'FOLD: ',i\n",
    "    print len(te_ind),len(tr_ind)\n",
    "    X_train,X_train_label = train[tr_ind],train_labels[tr_ind]\n",
    "    X_val,X_val_label = train[te_ind],train_labels[te_ind]\n",
    "    dtrain = lgb.Dataset(X_train,X_train_label) \n",
    "    dval   = lgb.Dataset(X_val,X_val_label, reference = dtrain)   \n",
    "    params = {\n",
    "            'task':'train', \n",
    "            'boosting_type':'gbdt',\n",
    "            'num_leaves': 63,\n",
    "            'objective': 'multiclass',\n",
    "            'num_class':8,\n",
    "            'learning_rate': 0.045,\n",
    "            'feature_fraction': 0.5,\n",
    "            'subsample':0.7,\n",
    "            'num_threads': 54,\n",
    "            'metric':'multi_logloss',\n",
    "            'seed':100\n",
    "        }  \n",
    "    model = lgb.train(params, dtrain, num_boost_round=100000,valid_sets=[dtrain,dval],verbose_eval=100, early_stopping_rounds=100)  \n",
    "    pred_val = model.predict(X_val)\n",
    "    pred_test = model.predict(test)\n",
    "    \n",
    "    meta_train[te_ind] = pred_val\n",
    "    meta_test += pred_test\n",
    "meta_test /= 5.0\n",
    "\n",
    "pd.to_pickle(meta_train,'../train_meta_lgb_4.pkl')\n",
    "pd.to_pickle(meta_test,'../test_meta_lgb_4.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
